{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "23fb2b49-3a93-4ac8-9f21-92e7ff7ac03e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b7b32108-b019-4526-8bb4-2721e6ef56b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e134624d-9690-4429-af0a-90dc78781b50",
   "metadata": {},
   "outputs": [],
   "source": [
    "float_data = pd.Series([1.2,-3.5,np.nan,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9a610bd3-fb0c-4912-90ba-03bb6575671c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.2\n",
       "1   -3.5\n",
       "2    NaN\n",
       "3    0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "78d1cb97-63a0-4e96-9cbc-a20ac878f416",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2     True\n",
       "3    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c3a2d7d1-f792-469c-b774-8f628cd75979",
   "metadata": {},
   "outputs": [],
   "source": [
    "string_data=pd.Series(['aardvark',np.nan,None,'avocado'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c7d44674-fe70-4748-9b06-d5e780ec6448",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    aardvark\n",
       "1         NaN\n",
       "2        None\n",
       "3     avocado\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1631a992-58f2-4338-abf0-f36b3a0bdb23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1     True\n",
       "2     True\n",
       "3    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c28f397e-1de3-4db3-b323-f63d7fae545d",
   "metadata": {},
   "outputs": [],
   "source": [
    "float_data=pd.Series([1,2,None],dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8777211a-fc09-4768-948c-4d96a4420da0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    2.0\n",
       "2    NaN\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f83d9486-779d-48a7-9591-ce9fa3983a1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3cdbcfb1-d77c-4698-876c-c2841008dc94",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.Series([1,np.nan,3.5,np.nan,7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "18709d71-1320-444a-90a6-e85b4c5cc41a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "2    3.5\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "72aa5e2b-95f0-46d3-8afa-68252959993f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "2    3.5\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data.notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ae93ef6e-d733-4e07-b7b0-68a366893ee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.DataFrame([[1.,6.5,3.],[1.,np.nan,np.nan],[np.nan,np.nan,np.nan],[np.nan,6.5,3.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "341c652c-27e2-4518-81b1-fba49b954dde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "2  NaN  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7294b4c5-4536-48c7-93be-4c43cc85cc70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "961bfd95-1955-4a7c-8f7c-bfd4ea499743",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "31ca5219-eed9-44ef-8143-72c7509bdaf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[4]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2942e3fc-e2e4-4ecb-93ee-e32dd089083a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2   4\n",
       "0  1.0  6.5  3.0 NaN\n",
       "1  1.0  NaN  NaN NaN\n",
       "2  NaN  NaN  NaN NaN\n",
       "3  NaN  6.5  3.0 NaN"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f49ce0a5-b383-4418-8e5b-9d082e3acf74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "2  NaN  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis='columns',how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d6b9d604-a550-4a2a-ab50-2432a5b0ebfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df =pd.DataFrame(np.random.standard_normal((7,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "dcd6c00f-90ef-4696-8835-d5d5bb53fb24",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:4,1] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2f639faf-4bdd-4831-9e72-bcb882fb10bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:2,2]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "99cb054d-46ba-460a-b132-852d7e1c723c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.054973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.570934</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.479752</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.020750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.624727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.149026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.198419</td>\n",
       "      <td>-0.768671</td>\n",
       "      <td>2.181003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.161829</td>\n",
       "      <td>-2.398304</td>\n",
       "      <td>1.132870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.593978</td>\n",
       "      <td>0.638091</td>\n",
       "      <td>0.049013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  1.054973       NaN       NaN\n",
       "1 -0.570934       NaN       NaN\n",
       "2  1.479752       NaN  1.020750\n",
       "3 -0.624727       NaN  1.149026\n",
       "4 -1.198419 -0.768671  2.181003\n",
       "5  0.161829 -2.398304  1.132870\n",
       "6  0.593978  0.638091  0.049013"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e5fc9cbb-e8b4-4fc9-a5eb-4cb90901e6fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.198419</td>\n",
       "      <td>-0.768671</td>\n",
       "      <td>2.181003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.161829</td>\n",
       "      <td>-2.398304</td>\n",
       "      <td>1.132870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.593978</td>\n",
       "      <td>0.638091</td>\n",
       "      <td>0.049013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "4 -1.198419 -0.768671  2.181003\n",
       "5  0.161829 -2.398304  1.132870\n",
       "6  0.593978  0.638091  0.049013"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "694745a0-27ea-49bb-a62b-cf16f36a8a34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.479752</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.020750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.624727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.149026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.198419</td>\n",
       "      <td>-0.768671</td>\n",
       "      <td>2.181003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.161829</td>\n",
       "      <td>-2.398304</td>\n",
       "      <td>1.132870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.593978</td>\n",
       "      <td>0.638091</td>\n",
       "      <td>0.049013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "2  1.479752       NaN  1.020750\n",
       "3 -0.624727       NaN  1.149026\n",
       "4 -1.198419 -0.768671  2.181003\n",
       "5  0.161829 -2.398304  1.132870\n",
       "6  0.593978  0.638091  0.049013"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(thresh=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1a8fb43b-a715-4cc5-8a1c-8eeec632585c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.054973</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.570934</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.479752</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.020750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.624727</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.149026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.198419</td>\n",
       "      <td>-0.768671</td>\n",
       "      <td>2.181003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.161829</td>\n",
       "      <td>-2.398304</td>\n",
       "      <td>1.132870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.593978</td>\n",
       "      <td>0.638091</td>\n",
       "      <td>0.049013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  1.054973  0.000000  0.000000\n",
       "1 -0.570934  0.000000  0.000000\n",
       "2  1.479752  0.000000  1.020750\n",
       "3 -0.624727  0.000000  1.149026\n",
       "4 -1.198419 -0.768671  2.181003\n",
       "5  0.161829 -2.398304  1.132870\n",
       "6  0.593978  0.638091  0.049013"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "283683bd-1e5f-4368-b7eb-1b5114a36a08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.054973</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.570934</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.479752</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.020750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.624727</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.149026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.198419</td>\n",
       "      <td>-0.768671</td>\n",
       "      <td>2.181003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.161829</td>\n",
       "      <td>-2.398304</td>\n",
       "      <td>1.132870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.593978</td>\n",
       "      <td>0.638091</td>\n",
       "      <td>0.049013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  1.054973  0.500000  0.000000\n",
       "1 -0.570934  0.500000  0.000000\n",
       "2  1.479752  0.500000  1.020750\n",
       "3 -0.624727  0.500000  1.149026\n",
       "4 -1.198419 -0.768671  2.181003\n",
       "5  0.161829 -2.398304  1.132870\n",
       "6  0.593978  0.638091  0.049013"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna({1:0.5,2:0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ff8aec04-e1f2-4250-926f-4e5d9df922a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(np.random.standard_normal((6,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2cc6f7b7-b112-4aad-8e08-58486174dee6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[2:,1]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3c54da08-6956-4dc8-9773-8806040a5b14",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[4:,2] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "97028659-026c-4cdd-9878-0b665679c63f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.601698</td>\n",
       "      <td>0.873698</td>\n",
       "      <td>-0.753073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.577215</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-0.481570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.487358</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.212411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.575388</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.179047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.583683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -1.601698  0.873698 -0.753073\n",
       "1  0.577215 -1.695172 -0.481570\n",
       "2 -1.487358       NaN  1.212411\n",
       "3  1.575388       NaN -1.162236\n",
       "4 -0.179047       NaN       NaN\n",
       "5 -1.583683       NaN       NaN"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b63f309b-f094-4f0e-b1bd-bfec7bd79879",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.601698</td>\n",
       "      <td>0.873698</td>\n",
       "      <td>-0.753073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.577215</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-0.481570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.487358</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>1.212411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.575388</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.179047</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.583683</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -1.601698  0.873698 -0.753073\n",
       "1  0.577215 -1.695172 -0.481570\n",
       "2 -1.487358 -1.695172  1.212411\n",
       "3  1.575388 -1.695172 -1.162236\n",
       "4 -0.179047 -1.695172 -1.162236\n",
       "5 -1.583683 -1.695172 -1.162236"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3d1844cc-31e9-4b96-bb02-51f3ab496038",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.601698</td>\n",
       "      <td>0.873698</td>\n",
       "      <td>-0.753073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.577215</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-0.481570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.487358</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>1.212411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.575388</td>\n",
       "      <td>-1.695172</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.179047</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.583683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.162236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -1.601698  0.873698 -0.753073\n",
       "1  0.577215 -1.695172 -0.481570\n",
       "2 -1.487358 -1.695172  1.212411\n",
       "3  1.575388 -1.695172 -1.162236\n",
       "4 -0.179047       NaN -1.162236\n",
       "5 -1.583683       NaN -1.162236"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.ffill(limit=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e48896fa-4f35-4305-94ab-d0583b515107",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series([1.,np.nan,3.5,np.nan,7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0d1ab972-ae09-4e5a-b87b-b4859cad1457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.000000\n",
       "1    3.833333\n",
       "2    3.500000\n",
       "3    3.833333\n",
       "4    7.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.fillna(data.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "eb2985b2-6b16-43b4-8cdf-c33626ef90fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.DataFrame({'k1':['one','two']*3+['two'],'k2':[1,1,2,3,3,4,4]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7ffc912d-b627-454d-9615-8f8ddba4d9d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2\n",
       "0  one   1\n",
       "1  two   1\n",
       "2  one   2\n",
       "3  two   3\n",
       "4  one   3\n",
       "5  two   4\n",
       "6  two   4"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "44c5916f-9a54-464b-b2ff-66be9821731b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5    False\n",
       "6     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8a9147db-5b03-4c96-9984-9fe62a42b84f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2\n",
       "0  one   1\n",
       "1  two   1\n",
       "2  one   2\n",
       "3  two   3\n",
       "4  one   3\n",
       "5  two   4"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7ceed7ae-5708-48af-bc96-542c69b52953",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['v1'] =range(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "7daad270-b358-4327-928a-1434c35d08b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1\n",
       "2  one   2   2\n",
       "3  two   3   3\n",
       "4  one   3   4\n",
       "5  two   4   5\n",
       "6  two   4   6"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "cdb761b8-dd24-417b-ac39-6a26104b51c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates(subset=['k1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9586d4a6-730c-49df-a4d8-aa0e45c502bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1\n",
       "2  one   2   2\n",
       "3  two   3   3\n",
       "4  one   3   4\n",
       "6  two   4   6"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates(['k1','k2'],keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "de91ac1b-bc6e-4912-a943-2b58abb6ff95",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.DataFrame({'food':[\"bacon\", \"pulled pork\", \"bacon\",\n",
    "                           \"pastrami\", \"corned beef\", \"bacon\",\n",
    "                           \"pastrami\", \"honey ham\", \"nova lox\"],\n",
    "                  \"ounces\":[4,3,12,6,7.5,8,3,5,6]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "be936bf1-9cc7-4cdf-8ac3-f4f0984b75b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>food</th>\n",
       "      <th>ounces</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bacon</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pulled pork</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bacon</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>corned beef</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bacon</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>honey ham</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>nova lox</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          food  ounces\n",
       "0        bacon     4.0\n",
       "1  pulled pork     3.0\n",
       "2        bacon    12.0\n",
       "3     pastrami     6.0\n",
       "4  corned beef     7.5\n",
       "5        bacon     8.0\n",
       "6     pastrami     3.0\n",
       "7    honey ham     5.0\n",
       "8     nova lox     6.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "22916a5b-d6d2-4aca-a7f7-88d99213094c",
   "metadata": {},
   "outputs": [],
   "source": [
    "meat_to_animal = {'bacon':'pig',\n",
    "                  'pulled pork':'pig',\n",
    "                  'pastrami':'cow',\n",
    "                  'corned beef':'cow',\n",
    "                  'honey ham':'pig',\n",
    "                  'nova lox':'salmon'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1965b22c-712e-4f7a-b2bd-f6bc146aa70f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['animal'] = data['food'].map(meat_to_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "8e2deff5-dcfc-4b78-a4ae-6ff113230bdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>food</th>\n",
       "      <th>ounces</th>\n",
       "      <th>animal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bacon</td>\n",
       "      <td>4.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pulled pork</td>\n",
       "      <td>3.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bacon</td>\n",
       "      <td>12.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>6.0</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>corned beef</td>\n",
       "      <td>7.5</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bacon</td>\n",
       "      <td>8.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>3.0</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>honey ham</td>\n",
       "      <td>5.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>nova lox</td>\n",
       "      <td>6.0</td>\n",
       "      <td>salmon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          food  ounces  animal\n",
       "0        bacon     4.0     pig\n",
       "1  pulled pork     3.0     pig\n",
       "2        bacon    12.0     pig\n",
       "3     pastrami     6.0     cow\n",
       "4  corned beef     7.5     cow\n",
       "5        bacon     8.0     pig\n",
       "6     pastrami     3.0     cow\n",
       "7    honey ham     5.0     pig\n",
       "8     nova lox     6.0  salmon"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "fc3a9beb-a01c-4477-bb71-46e5e4743854",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_animal(x):\n",
    "    return meat_to_animal[x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b6d41ce4-fb10-474d-b5be-d603c4206208",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       pig\n",
       "1       pig\n",
       "2       pig\n",
       "3       cow\n",
       "4       cow\n",
       "5       pig\n",
       "6       cow\n",
       "7       pig\n",
       "8    salmon\n",
       "Name: food, dtype: object"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['food'].map(get_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "a409f338-96ea-4252-a0ef-70acef85bd95",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.Series([1.,-999.,2.,-999.,-1000.,3.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "83a9bff0-42ba-44b9-a26b-d804f7be8468",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1.0\n",
       "1    -999.0\n",
       "2       2.0\n",
       "3    -999.0\n",
       "4   -1000.0\n",
       "5       3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "31e65bf4-c702-4338-a27f-e110a5094e43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1.0\n",
       "1       NaN\n",
       "2       2.0\n",
       "3       NaN\n",
       "4   -1000.0\n",
       "5       3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace(-999,np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "2a0a3629-9d15-427d-b01b-072101baae4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    NaN\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999,-1000],np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ed7b67b3-764b-4cd0-83f5-51cd0e78a725",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    0.0\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999,-1000],[np.nan,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "69c2bac3-5e86-4e8b-a8ab-03dde6f3e49f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    0.0\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace({-999:np.nan,-1000:0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ce2e98ed-1049-49b1-a436-49a6fedb02b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.DataFrame(np.arange(12).reshape((3,4)),\n",
    "                 index=[\"Ohio\", \"Colorado\", \"New York\"],\n",
    "                 columns=[\"one\", \"two\", \"three\", \"four\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "789fc1c8-af0e-47eb-853e-18fc2cfb40fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transfrom(x):\n",
    "    return x[:4].upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "5e12ec05-05fa-4260-bc20-3245bcb1e28e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['OHIO', 'COLO', 'NEW '], dtype='object')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.index.map(transfrom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "17beff8d-422f-42c7-8eef-b6fd85be84b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.index = data.index.map(transfrom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "01d44194-a769-4335-a6fe-0fd1fc9e3297",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>OHIO</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COLO</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEW</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      one  two  three  four\n",
       "OHIO    0    1      2     3\n",
       "COLO    4    5      6     7\n",
       "NEW     8    9     10    11"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "17308f7d-10da-47a8-8d73-e8f4326a6b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ONE</th>\n",
       "      <th>TWO</th>\n",
       "      <th>THREE</th>\n",
       "      <th>FOUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Ohio</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colo</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ONE  TWO  THREE  FOUR\n",
       "Ohio    0    1      2     3\n",
       "Colo    4    5      6     7\n",
       "New     8    9     10    11"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index=str.title,columns=str.upper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "9adfa2b0-8f72-4019-b24c-66940541d554",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>peekaboo</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>INDIANA</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COLO</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEW</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         one  two  peekaboo  four\n",
       "INDIANA    0    1         2     3\n",
       "COLO       4    5         6     7\n",
       "NEW        8    9        10    11"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index={'OHIO':'INDIANA'},columns={'three':'peekaboo'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5ab69f25-73e3-47de-b3c0-507a29d34158",
   "metadata": {},
   "outputs": [],
   "source": [
    "ages=[20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e5ec18b3-9442-4649-bfd2-a67ce75e5538",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=[18,25,35,60,100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d40c8ec9-bcb5-4f26-b9a7-7010193097b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "age_categories = pd.cut(ages,bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "52e60baf-c57e-4964-b51f-311d3abd116e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]\n",
       "Length: 12\n",
       "Categories (4, interval[int64, right]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "29536bab-f68c-4376-a27a-e1add42127e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "fa186809-fd78-4ba1-a53b-c13cecebdb90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]], dtype='interval[int64, right]')"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "211dbf86-805f-4ea3-8938-2ba6970ed866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Interval(18, 25, closed='right')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories.categories[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "14ec77af-8729-49c7-a53f-d6b2b76cbb79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BF\\AppData\\Local\\Temp\\ipykernel_11768\\3010498523.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(age_categories)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(18, 25]     5\n",
       "(25, 35]     3\n",
       "(35, 60]     3\n",
       "(60, 100]    1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(age_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "6af7209e-2848-468d-8ebb-ef1f0eafd5f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[18, 25), [18, 25), [25, 35), [25, 35), [18, 25), ..., [25, 35), [60, 100), [35, 60), [35, 60), [25, 35)]\n",
       "Length: 12\n",
       "Categories (4, interval[int64, left]): [[18, 25) < [25, 35) < [35, 60) < [60, 100)]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(ages,bins,right=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "0915da8e-902a-4f12-8afb-c0cec222a07d",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_names=['Youth','YoungAdult','MiddleAged','Senior']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "18c79dc6-fc2d-44d8-97e1-8184f82c921c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Youth', 'Youth', 'Youth', 'YoungAdult', 'Youth', ..., 'YoungAdult', 'Senior', 'MiddleAged', 'MiddleAged', 'YoungAdult']\n",
       "Length: 12\n",
       "Categories (4, object): ['Youth' < 'YoungAdult' < 'MiddleAged' < 'Senior']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(ages,bins,labels=group_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f8ad65fc-65d5-47b1-9a30-71f6a828c6e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=np.random.uniform(size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "303d7bb6-732e-4ac0-8103-c9b24e05cf1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.063, 0.28], (0.5, 0.71], (0.28, 0.5], (0.28, 0.5], (0.28, 0.5], ..., (0.063, 0.28], (0.5, 0.71], (0.063, 0.28], (0.28, 0.5], (0.063, 0.28]]\n",
       "Length: 20\n",
       "Categories (4, interval[float64, right]): [(0.063, 0.28] < (0.28, 0.5] < (0.5, 0.71] < (0.71, 0.93]]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(data,4,precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "551c9d94-868c-4447-9a43-53e2e456be57",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=np.random.standard_normal(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "16c41ab0-248c-48c8-a62e-6c7493a154d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "quartiles=pd.qcut(data,4,precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "350ebbd8-5aa6-46dc-b7f4-6ccd4d3c7a0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.7, 3.16], (-0.67, -0.015], (0.7, 3.16], (-0.015, 0.7], (-0.015, 0.7], ..., (-3.8299999999999996, -0.67], (-0.015, 0.7], (-3.8299999999999996, -0.67], (-0.015, 0.7], (-0.67, -0.015]]\n",
       "Length: 1000\n",
       "Categories (4, interval[float64, right]): [(-3.8299999999999996, -0.67] < (-0.67, -0.015] < (-0.015, 0.7] < (0.7, 3.16]]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quartiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "02b4878c-1f65-498d-8e47-c6567be19543",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BF\\AppData\\Local\\Temp\\ipykernel_11768\\3472704981.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(quartiles)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-3.8299999999999996, -0.67]    250\n",
       "(-0.67, -0.015]                 250\n",
       "(-0.015, 0.7]                   250\n",
       "(0.7, 3.16]                     250\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(quartiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "efc52dea-862c-41ee-8172-85a7f65286f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-3.819, -1.382]     100\n",
       "(-1.382, -0.0152]    400\n",
       "(-0.0152, 1.312]     400\n",
       "(1.312, 3.156]       100\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.qcut(data,[0,0.1,0.5,0.9,1.]).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "c145f31b-eed5-4b9c-b38c-0c7c979457fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.DataFrame(np.random.standard_normal((1000,4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b33f9ff6-df5c-4eb9-a801-fdc44696809b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.001407</td>\n",
       "      <td>0.026680</td>\n",
       "      <td>-0.024475</td>\n",
       "      <td>-0.002209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.989212</td>\n",
       "      <td>1.010933</td>\n",
       "      <td>1.027285</td>\n",
       "      <td>1.013384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.443157</td>\n",
       "      <td>-3.088141</td>\n",
       "      <td>-3.004006</td>\n",
       "      <td>-3.621318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.695678</td>\n",
       "      <td>-0.657111</td>\n",
       "      <td>-0.694786</td>\n",
       "      <td>-0.661937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.003138</td>\n",
       "      <td>0.074906</td>\n",
       "      <td>-0.021959</td>\n",
       "      <td>0.015368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.655826</td>\n",
       "      <td>0.726659</td>\n",
       "      <td>0.648831</td>\n",
       "      <td>0.659264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.077080</td>\n",
       "      <td>3.027623</td>\n",
       "      <td>3.180051</td>\n",
       "      <td>2.932393</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.001407     0.026680    -0.024475    -0.002209\n",
       "std       0.989212     1.010933     1.027285     1.013384\n",
       "min      -3.443157    -3.088141    -3.004006    -3.621318\n",
       "25%      -0.695678    -0.657111    -0.694786    -0.661937\n",
       "50%      -0.003138     0.074906    -0.021959     0.015368\n",
       "75%       0.655826     0.726659     0.648831     0.659264\n",
       "max       3.077080     3.027623     3.180051     2.932393"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "ef702095-d8ea-4fd2-8201-9cd50b2e212c",
   "metadata": {},
   "outputs": [],
   "source": [
    "col=data[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e0510d94-4e8b-4ffa-89e5-23685e5f9d5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123    3.180051\n",
       "300    3.083119\n",
       "779    3.110767\n",
       "803    3.063395\n",
       "856   -3.004006\n",
       "Name: 2, dtype: float64"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col[col.abs()>3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "234481b4-bd47-4ea9-a0ca-592cb1801ef6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.388389</td>\n",
       "      <td>-0.317532</td>\n",
       "      <td>-1.525090</td>\n",
       "      <td>-3.364115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>-3.443157</td>\n",
       "      <td>-0.918489</td>\n",
       "      <td>0.080527</td>\n",
       "      <td>0.268312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>1.588819</td>\n",
       "      <td>1.594291</td>\n",
       "      <td>3.180051</td>\n",
       "      <td>1.702383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>1.452373</td>\n",
       "      <td>-3.080009</td>\n",
       "      <td>1.290158</td>\n",
       "      <td>0.024872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>1.595489</td>\n",
       "      <td>0.269362</td>\n",
       "      <td>0.074987</td>\n",
       "      <td>-3.621318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>1.342794</td>\n",
       "      <td>0.670258</td>\n",
       "      <td>3.083119</td>\n",
       "      <td>-1.579857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>-0.189584</td>\n",
       "      <td>-3.088141</td>\n",
       "      <td>1.787341</td>\n",
       "      <td>-0.675498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>3.077080</td>\n",
       "      <td>0.382461</td>\n",
       "      <td>-0.130284</td>\n",
       "      <td>0.145526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>-0.649327</td>\n",
       "      <td>1.214832</td>\n",
       "      <td>3.110767</td>\n",
       "      <td>-0.503137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>1.881349</td>\n",
       "      <td>-0.789320</td>\n",
       "      <td>3.063395</td>\n",
       "      <td>0.345373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>1.286918</td>\n",
       "      <td>0.034543</td>\n",
       "      <td>-1.377777</td>\n",
       "      <td>-3.305900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>-0.034303</td>\n",
       "      <td>1.968628</td>\n",
       "      <td>-3.004006</td>\n",
       "      <td>-0.840449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>905</th>\n",
       "      <td>-0.292211</td>\n",
       "      <td>3.027623</td>\n",
       "      <td>-1.943574</td>\n",
       "      <td>1.427279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3\n",
       "13  -0.388389 -0.317532 -1.525090 -3.364115\n",
       "118 -3.443157 -0.918489  0.080527  0.268312\n",
       "123  1.588819  1.594291  3.180051  1.702383\n",
       "205  1.452373 -3.080009  1.290158  0.024872\n",
       "255  1.595489  0.269362  0.074987 -3.621318\n",
       "300  1.342794  0.670258  3.083119 -1.579857\n",
       "448 -0.189584 -3.088141  1.787341 -0.675498\n",
       "460  3.077080  0.382461 -0.130284  0.145526\n",
       "779 -0.649327  1.214832  3.110767 -0.503137\n",
       "803  1.881349 -0.789320  3.063395  0.345373\n",
       "855  1.286918  0.034543 -1.377777 -3.305900\n",
       "856 -0.034303  1.968628 -3.004006 -0.840449\n",
       "905 -0.292211  3.027623 -1.943574  1.427279"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[(data.abs()>3).any(axis='columns')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "aeea8261-db69-4b57-859d-fe9bd16853e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[data.abs()>3] = np.sign(data)*3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "5fa2cd13-267f-4d66-9b56-e9b42082ae58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.001773</td>\n",
       "      <td>0.026821</td>\n",
       "      <td>-0.024909</td>\n",
       "      <td>-0.000918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.987528</td>\n",
       "      <td>1.010340</td>\n",
       "      <td>1.025956</td>\n",
       "      <td>1.009249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.695678</td>\n",
       "      <td>-0.657111</td>\n",
       "      <td>-0.694786</td>\n",
       "      <td>-0.661937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.003138</td>\n",
       "      <td>0.074906</td>\n",
       "      <td>-0.021959</td>\n",
       "      <td>0.015368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.655826</td>\n",
       "      <td>0.726659</td>\n",
       "      <td>0.648831</td>\n",
       "      <td>0.659264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.932393</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.001773     0.026821    -0.024909    -0.000918\n",
       "std       0.987528     1.010340     1.025956     1.009249\n",
       "min      -3.000000    -3.000000    -3.000000    -3.000000\n",
       "25%      -0.695678    -0.657111    -0.694786    -0.661937\n",
       "50%      -0.003138     0.074906    -0.021959     0.015368\n",
       "75%       0.655826     0.726659     0.648831     0.659264\n",
       "max       3.000000     3.000000     3.000000     2.932393"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3e4f59ce-7853-49ab-be9f-3f3d219cce2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3\n",
       "0  1.0  1.0 -1.0  1.0\n",
       "1 -1.0  1.0 -1.0  1.0\n",
       "2  1.0  1.0  1.0  1.0\n",
       "3  1.0 -1.0  1.0  1.0\n",
       "4  1.0  1.0 -1.0  1.0"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sign(data).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "63ad5316-f45c-4340-965b-b1c776f5f715",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(np.arange(5*7).reshape((5,7)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "8907e3f7-521c-4e92-943d-482a7b04c045",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "0   0   1   2   3   4   5   6\n",
       "1   7   8   9  10  11  12  13\n",
       "2  14  15  16  17  18  19  20\n",
       "3  21  22  23  24  25  26  27\n",
       "4  28  29  30  31  32  33  34"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "7e6cefbe-4cb3-4c5f-9b42-60620ac1bb01",
   "metadata": {},
   "outputs": [],
   "source": [
    "sampler = np.random.permutation(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "a8011379-adca-4f7a-aee4-8f732715f9c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 2, 4, 0])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "33a09369-52d7-456c-80d6-85ddafbf8cf9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "1   7   8   9  10  11  12  13\n",
       "3  21  22  23  24  25  26  27\n",
       "2  14  15  16  17  18  19  20\n",
       "4  28  29  30  31  32  33  34\n",
       "0   0   1   2   3   4   5   6"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(sampler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "a2c37ac0-36b3-443a-9be8-4683c7abd419",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "1   7   8   9  10  11  12  13\n",
       "3  21  22  23  24  25  26  27\n",
       "2  14  15  16  17  18  19  20\n",
       "4  28  29  30  31  32  33  34\n",
       "0   0   1   2   3   4   5   6"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[sampler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "db3c78f0-0053-4536-8dbf-795fe1334da6",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_sampler=np.random.permutation(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "549add5f-b977-482f-9361-8760891787ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 6, 4, 2, 0, 5, 3])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "0da324a8-1af2-4e41-ab93-f827072a7002",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>6</th>\n",
       "      <th>4</th>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <th>5</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>18</td>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>23</td>\n",
       "      <td>21</td>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29</td>\n",
       "      <td>34</td>\n",
       "      <td>32</td>\n",
       "      <td>30</td>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    1   6   4   2   0   5   3\n",
       "0   1   6   4   2   0   5   3\n",
       "1   8  13  11   9   7  12  10\n",
       "2  15  20  18  16  14  19  17\n",
       "3  22  27  25  23  21  26  24\n",
       "4  29  34  32  30  28  33  31"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(column_sampler,axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "8d339aa5-2728-4ac4-bc33-7493ea4a5353",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "4  28  29  30  31  32  33  34\n",
       "3  21  22  23  24  25  26  27\n",
       "0   0   1   2   3   4   5   6"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "4ada73b8-7e65-4436-914a-b7f3016ad3b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "choices=pd.Series([5,7,-1,6,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "7ab78b3b-fddf-4362-9136-c9e055ac8edb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    6\n",
       "2   -1\n",
       "1    7\n",
       "3    6\n",
       "4    4\n",
       "4    4\n",
       "2   -1\n",
       "1    7\n",
       "2   -1\n",
       "1    7\n",
       "dtype: int64"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "choices.sample(n=10,replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "1d297c0a-cd8f-4c1e-855f-472a16c7ee00",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame({\"key\":['b','b','a','c','a','b'],'data1':range(6)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5e04a27f-c8eb-4a3d-845b-48a7bc582d41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key  data1\n",
       "0   b      0\n",
       "1   b      1\n",
       "2   a      2\n",
       "3   c      3\n",
       "4   a      4\n",
       "5   b      5"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "806dddf7-778e-4011-bd09-f64d52a4205a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a    b    c\n",
       "0  0.0  1.0  0.0\n",
       "1  0.0  1.0  0.0\n",
       "2  1.0  0.0  0.0\n",
       "3  0.0  0.0  1.0\n",
       "4  1.0  0.0  0.0\n",
       "5  0.0  1.0  0.0"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(df['key'],dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "c6a7008a-c09e-409d-bd3c-471cb5295fb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies=pd.get_dummies(df['key'],prefix='key',dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "15d2d297-d6c8-4a4b-9c0c-ffc7d3acd70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_with_dummy = df[['data1']].join(dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "5d3d3cb2-be35-40ff-8d38-ce7703989311",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>key_a</th>\n",
       "      <th>key_b</th>\n",
       "      <th>key_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   data1  key_a  key_b  key_c\n",
       "0      0    0.0    1.0    0.0\n",
       "1      1    0.0    1.0    0.0\n",
       "2      2    1.0    0.0    0.0\n",
       "3      3    0.0    0.0    1.0\n",
       "4      4    1.0    0.0    0.0\n",
       "5      5    0.0    1.0    0.0"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "9bd5f256-a513-454d-9db4-c241fbb5c8da",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnames=['movie_id','title','genres']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "bceb8ae1-eb82-43df-bc50-ab40efef7aa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies=pd.read_table('datasets/movielens/movies.dat',sep='::',header=None,names=mnames,engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "38f12fbd-585b-4f64-9a7c-cbda363fce92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id                               title                        genres\n",
       "0         1                    Toy Story (1995)   Animation|Children's|Comedy\n",
       "1         2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "2         3             Grumpier Old Men (1995)                Comedy|Romance\n",
       "3         4            Waiting to Exhale (1995)                  Comedy|Drama\n",
       "4         5  Father of the Bride Part II (1995)                        Comedy\n",
       "5         6                         Heat (1995)         Action|Crime|Thriller\n",
       "6         7                      Sabrina (1995)                Comedy|Romance\n",
       "7         8                 Tom and Huck (1995)          Adventure|Children's\n",
       "8         9                 Sudden Death (1995)                        Action\n",
       "9        10                    GoldenEye (1995)     Action|Adventure|Thriller"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "f5e72891-dfb9-489e-ac5d-3a5669b7aad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies=movies['genres'].str.get_dummies('|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "1f2de743-3f29-4e48-9952-6dd386a8c831",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children's</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Crime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Action  Adventure  Animation  Children's  Comedy  Crime\n",
       "0       0          0          1           1       1      0\n",
       "1       0          1          0           1       0      0\n",
       "2       0          0          0           0       1      0\n",
       "3       0          0          0           0       1      0\n",
       "4       0          0          0           0       1      0\n",
       "5       1          0          0           0       0      1\n",
       "6       0          0          0           0       1      0\n",
       "7       0          1          0           1       0      0\n",
       "8       1          0          0           0       0      0\n",
       "9       1          1          0           0       0      0"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.iloc[:10,:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "4d230092-67a7-43a7-aff6-ad0f4a22d916",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_windic = movies.join(dummies.add_prefix('Genre_'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "d1801fb6-4459-4956-b53e-35b2b89884ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "movie_id                                       1\n",
       "title                           Toy Story (1995)\n",
       "genres               Animation|Children's|Comedy\n",
       "Genre_Action                                   0\n",
       "Genre_Adventure                                0\n",
       "Genre_Animation                                1\n",
       "Genre_Children's                               1\n",
       "Genre_Comedy                                   1\n",
       "Genre_Crime                                    0\n",
       "Genre_Documentary                              0\n",
       "Genre_Drama                                    0\n",
       "Genre_Fantasy                                  0\n",
       "Genre_Film-Noir                                0\n",
       "Genre_Horror                                   0\n",
       "Genre_Musical                                  0\n",
       "Genre_Mystery                                  0\n",
       "Genre_Romance                                  0\n",
       "Genre_Sci-Fi                                   0\n",
       "Genre_Thriller                                 0\n",
       "Genre_War                                      0\n",
       "Genre_Western                                  0\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_windic.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "a2fb9f11-89ca-4e5c-94ff-a0caa9a909f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "a037de60-2c43-45c9-8bce-1e24fb85f291",
   "metadata": {},
   "outputs": [],
   "source": [
    "values=np.random.uniform(size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "0bdf884d-35f1-4bb9-9e40-f2f15a6612fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.92961609, 0.31637555, 0.18391881, 0.20456028, 0.56772503,\n",
       "       0.5955447 , 0.96451452, 0.6531771 , 0.74890664, 0.65356987])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "e92e6f00-10be-4fca-a5ef-a1c77cd13495",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=[0,0.2,0.4,0.6,0.8,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "7e782764-c831-4195-bbe7-56e47ff6b24f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>(0.0, 0.2]</th>\n",
       "      <th>(0.2, 0.4]</th>\n",
       "      <th>(0.4, 0.6]</th>\n",
       "      <th>(0.6, 0.8]</th>\n",
       "      <th>(0.8, 1.0]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   (0.0, 0.2]  (0.2, 0.4]  (0.4, 0.6]  (0.6, 0.8]  (0.8, 1.0]\n",
       "0       False       False       False       False        True\n",
       "1       False        True       False       False       False\n",
       "2        True       False       False       False       False\n",
       "3       False        True       False       False       False\n",
       "4       False       False        True       False       False\n",
       "5       False       False        True       False       False\n",
       "6       False       False       False       False        True\n",
       "7       False       False       False        True       False\n",
       "8       False       False       False        True       False\n",
       "9       False       False       False        True       False"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(pd.cut(values,bins))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "b4f2b688-b294-4511-878e-377c831c7c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.Series([1,2,3,None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "e9c99142-031d-41c8-8d85-7b1786578f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    2.0\n",
       "2    3.0\n",
       "3    NaN\n",
       "dtype: float64"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "9a4f12f5-b772-455f-b5d3-183c8520ffd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "2f5d00ca-9f82-4e95-baad-2213c895842b",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.Series([1,2,3,None],dtype=pd.Int64Dtype())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "6987714e-b5a8-441f-93a9-0073fd15621a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1\n",
       "1       2\n",
       "2       3\n",
       "3    <NA>\n",
       "dtype: Int64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "c6082c30-6a83-4ba3-a945-91d3a9b81091",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "3ae7236f-6aef-4daa-bdf6-2e62c26a7629",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Dtype()"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "967eea28-6bb1-405f-998a-f33f6c589801",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<NA>"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "374c9d70-f49a-4875-bb68-7fa384ba024a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[3] is pd.NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "cd35398b-5666-427d-9c82-5e85902b33c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.Series([1,2,3,None],dtype='Int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "3f01cecd-d269-4d07-9e3e-f94e226cb04c",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.Series(['one','two',None,'three'],dtype=pd.StringDtype())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "8d922568-80fb-4dd4-bd2e-d729426543a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      one\n",
       "1      two\n",
       "2     <NA>\n",
       "3    three\n",
       "dtype: string"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "7e36df09-26b7-413d-97c2-12b74cc76e2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame({'A':[1,2,None,4],\n",
    "                'B':['one','two','three',None],\n",
    "                'C':[False,None,False,True]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "76ef8b2b-1644-4411-9ca4-942731330604",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>one</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>two</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>three</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A      B      C\n",
       "0  1.0    one  False\n",
       "1  2.0    two   None\n",
       "2  NaN  three  False\n",
       "3  4.0   None   True"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "581cc958-a5bb-41c0-a431-e3de2a1e7c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['A'] = df['A'].astype('Int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "1ba8c6c1-d445-42c9-8e16-78ca3381bcf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['B'] =df['B'].astype('string')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "c8fcb6aa-45da-4ea5-882d-c82cf62f6212",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['C']=df['C'].astype('boolean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "1d9050e7-a82c-4c0e-af92-7f0f83877187",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>one</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>two</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>three</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      A      B      C\n",
       "0     1    one  False\n",
       "1     2    two   <NA>\n",
       "2  <NA>  three  False\n",
       "3     4   <NA>   True"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "813e6f0c-19fa-4562-b24a-7ece5d3b12a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "val='a,b,  guido'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "ed44c062-b9dd-4b5a-936b-bf418af4072e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'b', '  guido']"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "db59ebe1-3142-48d5-a277-881655d583be",
   "metadata": {},
   "outputs": [],
   "source": [
    "pieces=[x.strip() for x in val.split(',')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "977def58-1553-4723-8b1b-7ae814da5de4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'b', 'guido']"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "99bd292f-1b54-4ceb-882f-82b1b29c6e68",
   "metadata": {},
   "outputs": [],
   "source": [
    "first,second,third=pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "1b4f3913-9397-466a-be7a-b71ab0af8c66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::guido'"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first+\"::\"+second+\"::\"+third"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "03746307-d193-4f69-8e65-17a71e9a1116",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::guido'"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"::\".join(pieces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "3bbec36b-f1d8-4884-b638-70c53e91db90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'guido' in val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "fde001e6-9313-4e2c-bc4a-8e30876d9372",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.index(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "707e8cb3-23e0-44ae-914b-5a951392456f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.find(':')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "b8d4a8bf-91ab-4741-8ece-907b804af7f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.count(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "69bb077e-0873-484d-9013-bcaaf6859de5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::  guido'"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(',','::')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "38bd396f-199e-4a8e-a4e2-dda141ac9d6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ab  guido'"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(',','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "06ffc635-655a-4709-ab01-f9f9b36673ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "0b36d648-bedf-4ff1-84a8-3bfe64c92c4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "text='foo    bar\\t baz  \\tqux'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "5fdf9239-5242-4391-8f0c-f74f49331f2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'qux']"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'\\s+',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "c231ffd5-2f29-4977-9987-16e06bbb380a",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex=re.compile(r'\\s+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "906e41b4-9e64-4d19-ad84-d2de8e2f34d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'qux']"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.split(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "91834d26-24d1-477e-85b6-b69e2e39334c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['    ', '\\t ', '  \\t']"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "6d75a507-e8f3-4565-b557-83c4318f6a1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"Dave dave@google.com\n",
    "Steve steve@gmail.com\n",
    "Rob rob@gmail.com\n",
    "Ryan ryan@yahoo.com\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "70eee2e1-066d-4c6d-a142-f413ceeb33cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = r\"[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "db8bbd1e-8266-45b9-b828-0aa37bda7eb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex=re.compile(pattern,flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "4665149a-efa2-4fed-adff-1b9904e2b5ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dave@google.com', 'steve@gmail.com', 'rob@gmail.com', 'ryan@yahoo.com']"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "e3ddef43-406c-4285-b367-0fec44d1e69d",
   "metadata": {},
   "outputs": [],
   "source": [
    "m=regex.search(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "def202a9-1f20-47a1-8cfe-15b254c8adbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<re.Match object; span=(5, 20), match='dave@google.com'>"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "ae7ef103-02f2-44c4-9ed5-2200ee5432f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dave@google.com'"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[m.start():m.end()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "e6fbf294-6030-4196-8a6d-8c95ef62a8d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(regex.match(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "3c88584f-b029-42ff-946a-e0b75d8818a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave REPLACED\n",
      "Steve REPLACED\n",
      "Rob REPLACED\n",
      "Ryan REPLACED\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub('REPLACED',text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "35fd223a-4f52-4af8-b416-b574b518a678",
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "d9b690d1-33ee-4b50-baa4-e8473a8248b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile(pattern,flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "329393da-a0d9-44bd-bf2f-8f54afa5b954",
   "metadata": {},
   "outputs": [],
   "source": [
    "m=regex.match('wesm@bright.net')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "9b9a0b35-2e50-4081-bb2b-d6d27108afbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('wesm', 'bright', 'net')"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.groups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "7aae1f24-2eec-4c54-88b6-bfb468ce3837",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dave', 'google', 'com'),\n",
       " ('steve', 'gmail', 'com'),\n",
       " ('rob', 'gmail', 'com'),\n",
       " ('ryan', 'yahoo', 'com')]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "a66d04bc-18f3-42ac-ae2c-65412bb84f7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave Username:dave,Domain:google,Suffix:com\n",
      "Steve Username:steve,Domain:gmail,Suffix:com\n",
      "Rob Username:rob,Domain:gmail,Suffix:com\n",
      "Ryan Username:ryan,Domain:yahoo,Suffix:com\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub(r'Username:\\1,Domain:\\2,Suffix:\\3',text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "2e51e750-d4e6-421b-b720-0447c1eff3e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data={\"Dave\": \"dave@google.com\", \"Steve\": \"steve@gmail.com\",\n",
    "           \"Rob\": \"rob@gmail.com\", \"Wes\": np.nan}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "a72e4512-e3c2-4afd-a1ee-016cd9cad90a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "1339a92e-9ad5-43ce-a23a-5339d3e24f80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@google.com\n",
       "Steve    steve@gmail.com\n",
       "Rob        rob@gmail.com\n",
       "Wes                  NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "95681fb9-5c6e-4a30-8813-aafbe4f26cc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve    False\n",
       "Rob      False\n",
       "Wes       True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "3d17bf78-9cef-45e7-b2b5-6d29bf95ed09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve     True\n",
       "Rob       True\n",
       "Wes        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.contains('gmail')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "5244b522-9b2a-4217-b4a0-6f857116cf83",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_as_string_ext = data.astype('string')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "2fce82ea-39e6-4035-b9f2-a866761e6759",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@google.com\n",
       "Steve    steve@gmail.com\n",
       "Rob        rob@gmail.com\n",
       "Wes                 <NA>\n",
       "dtype: string"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "9443c530-7bd6-4a29-a2d6-c23afea202cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve     True\n",
       "Rob       True\n",
       "Wes       <NA>\n",
       "dtype: boolean"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext.str.contains('gmail')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "f164df06-5bcb-4dad-ac7d-95cb48e4d34a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "0d712358-a1f5-4fa8-b872-519ae55b41e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     [(dave, google, com)]\n",
       "Steve    [(steve, gmail, com)]\n",
       "Rob        [(rob, gmail, com)]\n",
       "Wes                        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.findall(pattern,flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "9d1e503e-201f-441c-a8d8-d78e70e19795",
   "metadata": {},
   "outputs": [],
   "source": [
    "matches =data.str.findall(pattern,flags=re.IGNORECASE).str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "75cec992-7863-4529-9e2e-2da783aa3919",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     (dave, google, com)\n",
       "Steve    (steve, gmail, com)\n",
       "Rob        (rob, gmail, com)\n",
       "Wes                      NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "82739981-8a57-4435-9eed-0d5f1f2a0f50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     google\n",
       "Steve     gmail\n",
       "Rob       gmail\n",
       "Wes         NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches.str.get(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "9d4a41b3-a60e-4aef-b66d-fa56d2929699",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@\n",
       "Steve    steve\n",
       "Rob      rob@g\n",
       "Wes        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "c193631f-79ff-48ac-a42c-a1ef0f9decc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Dave</th>\n",
       "      <td>dave</td>\n",
       "      <td>google</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Steve</th>\n",
       "      <td>steve</td>\n",
       "      <td>gmail</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rob</th>\n",
       "      <td>rob</td>\n",
       "      <td>gmail</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wes</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0       1    2\n",
       "Dave    dave  google  com\n",
       "Steve  steve   gmail  com\n",
       "Rob      rob   gmail  com\n",
       "Wes      NaN     NaN  NaN"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.extract(pattern,flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "153090cf-70e7-46d5-bc71-95afeb6ac721",
   "metadata": {},
   "outputs": [],
   "source": [
    "values = pd.Series(['apple','orange','apple','apple']*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "6599daf3-03db-4106-86a9-e2cad53e2fe5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "dtype: object"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "9c6ecf6e-436e-4981-aa84-e3bbc0430c4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['apple', 'orange'], dtype=object)"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "005688f3-4706-4bac-a77f-c0f9052235a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BF\\AppData\\Local\\Temp\\ipykernel_11768\\3297668723.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(values)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "apple     6\n",
       "orange    2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "cc093ec7-13ea-443e-b9b3-d2e180ecf90b",
   "metadata": {},
   "outputs": [],
   "source": [
    "values=pd.Series([0,1,0,0]*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "cbedde24-ca48-4345-80c0-60cb23c74031",
   "metadata": {},
   "outputs": [],
   "source": [
    "dim=pd.Series(['apple','orange'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "4da5983f-fd98-4874-99d4-5a227e87f958",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "5    1\n",
       "6    0\n",
       "7    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "4610a4a7-6a60-404b-a011-4a1b3e662667",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "dtype: object"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "21473057-9e6c-45ca-ae93-4b680ba81f4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "0     apple\n",
       "0     apple\n",
       "0     apple\n",
       "1    orange\n",
       "0     apple\n",
       "0     apple\n",
       "dtype: object"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim.take(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "316689b7-6b8d-4797-96f5-a71c5507efed",
   "metadata": {},
   "outputs": [],
   "source": [
    "fruits=['apple','orange','apple','apple']*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "190d5d49-63e5-4e90-88e2-44af0b02a56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = len(fruits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "a1236d1e-2208-4704-916d-9d1f07b2a2f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(seed=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "ba7b39f4-29bd-4f5b-9186-fca524732a4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'fruit':fruits,\n",
    "                  'basket_id':np.arange(N),\n",
    "                  'count':rng.integers(3,15,size=N),\n",
    "                  'weight':rng.uniform(0,14,size=N)},\n",
    "                  columns=['basket_id','fruit','count','weight'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "524febbf-0991-4a05-b2da-8e18ca0cce0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>basket_id</th>\n",
       "      <th>fruit</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>apple</td>\n",
       "      <td>11</td>\n",
       "      <td>5.475534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>orange</td>\n",
       "      <td>5</td>\n",
       "      <td>4.659395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>apple</td>\n",
       "      <td>12</td>\n",
       "      <td>8.376323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>apple</td>\n",
       "      <td>6</td>\n",
       "      <td>2.614279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>apple</td>\n",
       "      <td>5</td>\n",
       "      <td>9.418585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>orange</td>\n",
       "      <td>12</td>\n",
       "      <td>13.185240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>apple</td>\n",
       "      <td>10</td>\n",
       "      <td>3.475440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>apple</td>\n",
       "      <td>11</td>\n",
       "      <td>13.284336</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   basket_id   fruit  count     weight\n",
       "0          0   apple     11   5.475534\n",
       "1          1  orange      5   4.659395\n",
       "2          2   apple     12   8.376323\n",
       "3          3   apple      6   2.614279\n",
       "4          4   apple      5   9.418585\n",
       "5          5  orange     12  13.185240\n",
       "6          6   apple     10   3.475440\n",
       "7          7   apple     11  13.284336"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "eac9e585-c834-4874-aa5e-cf95be0c80df",
   "metadata": {},
   "outputs": [],
   "source": [
    "fruit_cat = df['fruit'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "ac6273f4-3f5e-4376-80e5-18af5f12d238",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "Name: fruit, dtype: category\n",
       "Categories (2, object): ['apple', 'orange']"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fruit_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "30965934-2905-4643-97a7-511af92f24fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = fruit_cat.array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "2f80f97b-8ee0-4913-a02e-a25a4ff820ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.arrays.categorical.Categorical"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "d9bd822f-17d9-4b1c-b99e-a5946d8555f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['apple', 'orange'], dtype='object')"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "a1265990-f79f-4cf9-b057-da8465e8ff57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 0, 1, 0, 0], dtype=int8)"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "df170c39-b7cc-4c90-9689-ca26f51ce3f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'apple', 1: 'orange'}"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(enumerate(c.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "79040443-f3e0-4bf9-a6cd-4cf604c5f27c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['fruit'] = df['fruit'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "69dcbd9e-12cb-4f0d-a199-4c88a2b165de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "Name: fruit, dtype: category\n",
       "Categories (2, object): ['apple', 'orange']"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['fruit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "b5d6688a-deb4-4957-9a83-d42ce36b2cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_categories=pd.Categorical(['foo','bar','baz','foo','bar'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "9c76dca4-57a3-48c2-9be8-d99a1d99973a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'bar']\n",
       "Categories (3, object): ['bar', 'baz', 'foo']"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "de897e0f-b7ba-4c56-b969-7db637727d98",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories=['foo','bar','baz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "a464f42e-1b69-4f87-b37f-5f9091657190",
   "metadata": {},
   "outputs": [],
   "source": [
    "codes=[0,1,2,0,0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "994a8175-270d-47e1-9a31-1bb7f28b0570",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_cats_2 = pd.Categorical.from_codes(codes,categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "990bf1d1-47cc-48ea-9909-efc87ed774b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo', 'bar', 'baz']"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "b9d1d79f-ec40-4ce0-b15e-f46f0908327e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordered_cat = pd.Categorical.from_codes(codes,categories,ordered=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "66fa76f2-309f-4cd2-8360-fb754afe1087",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo' < 'bar' < 'baz']"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "861984c9-17d5-4c63-a827-5db4f40fd895",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo' < 'bar' < 'baz']"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "2a5c57fd-b1c3-4d20-a4a2-9702ce76914d",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng=np.random.default_rng(seed=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "200edd93-939e-4a4f-8b7d-345f4004570c",
   "metadata": {},
   "outputs": [],
   "source": [
    "draws = rng.standard_normal(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "608db913-e8f3-49f0-b7dc-67f8eac789d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.42382504,  1.26372846, -0.87066174, -0.25917323, -0.07534331])"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draws[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "e85e4317-e2d8-4bdc-829c-a28258d5c280",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins= pd.qcut(draws,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "fab81780-6b13-4a77-a741-5198cc646a16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(-3.121, -0.675], (0.687, 3.211], (-3.121, -0.675], (-0.675, 0.0134], (-0.675, 0.0134], ..., (0.0134, 0.687], (0.0134, 0.687], (-0.675, 0.0134], (0.0134, 0.687], (-0.675, 0.0134]]\n",
       "Length: 1000\n",
       "Categories (4, interval[float64, right]): [(-3.121, -0.675] < (-0.675, 0.0134] < (0.0134, 0.687] < (0.687, 3.211]]"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "8ec3c04a-2819-4db9-843c-08a1311bb925",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=pd.qcut(draws,4,labels=['Q1','Q2','Q3','Q4'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "a548a48e-e063-4878-8d34-b585fdca6269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Q1', 'Q4', 'Q1', 'Q2', 'Q2', ..., 'Q3', 'Q3', 'Q2', 'Q3', 'Q2']\n",
       "Length: 1000\n",
       "Categories (4, object): ['Q1' < 'Q2' < 'Q3' < 'Q4']"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "d00079fe-4dbb-434b-a677-fe2b44c17c57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 3, 0, 1, 1, 0, 0, 2, 2, 0], dtype=int8)"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins.codes[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "id": "c71e684b-8406-45d3-9f09-a6243856d69e",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins= pd.Series(bins,name='quartile')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "2e451d3a-9386-48e6-8875-e9dbb8ef7e2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = (pd.Series(draws).groupby(bins,observed=False).agg(['count','min','max']).reset_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "61aed2ff-4021-4f2b-ab52-677ac3f6f6f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quartile</th>\n",
       "      <th>count</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>250</td>\n",
       "      <td>-3.119609</td>\n",
       "      <td>-0.678494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>250</td>\n",
       "      <td>-0.673305</td>\n",
       "      <td>0.008009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q3</td>\n",
       "      <td>250</td>\n",
       "      <td>0.018753</td>\n",
       "      <td>0.686183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q4</td>\n",
       "      <td>250</td>\n",
       "      <td>0.688282</td>\n",
       "      <td>3.211418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  quartile  count       min       max\n",
       "0       Q1    250 -3.119609 -0.678494\n",
       "1       Q2    250 -0.673305  0.008009\n",
       "2       Q3    250  0.018753  0.686183\n",
       "3       Q4    250  0.688282  3.211418"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "239eb000-634f-4636-9db4-06f705063009",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Q1\n",
       "1    Q2\n",
       "2    Q3\n",
       "3    Q4\n",
       "Name: quartile, dtype: category\n",
       "Categories (4, object): ['Q1' < 'Q2' < 'Q3' < 'Q4']"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['quartile']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "5acf1e11-dacf-4667-aaa3-5da83f836778",
   "metadata": {},
   "outputs": [],
   "source": [
    "N=10_000_000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "6eb0e1bc-ac55-48af-a886-77b28fdf262b",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=pd.Series(['foo','bar','baz','qux']*(N//4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "f9c0ac30-428b-45b4-af03-cc8b8aff8094",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "8cdb4073-75ea-494f-9e49-dd0b7c833ae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "600000128"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "e6af367c-d4b9-49d4-9cfb-b34693d137ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000540"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "c9ca8edd-925d-4bf5-9a54-c4ef005ad423",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 391 ms\n",
      "Wall time: 395 ms\n"
     ]
    }
   ],
   "source": [
    "%time _ = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "ebed50a5-b569-4e64-ac78-8aab298bf9d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "347 ms ± 7.99 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%timeit labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "3f66ffc4-54bd-42c0-afcf-fd3eaba99f9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40 ms ± 4.4 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit categories.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "9eefecde-e1d4-4005-a9c7-00ce2061cc6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.Series(['a','b','c','d']*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "1ddd0be3-4b3b-4cc7-80e5-57d5ff030978",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s=s.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "03ea148d-96d4-4f40-80ae-99a54fa847cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "2    c\n",
       "3    d\n",
       "4    a\n",
       "5    b\n",
       "6    c\n",
       "7    d\n",
       "dtype: category\n",
       "Categories (4, object): ['a', 'b', 'c', 'd']"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "825d80ed-70e7-4f7a-9150-410fb8e9421e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    2\n",
       "3    3\n",
       "4    0\n",
       "5    1\n",
       "6    2\n",
       "7    3\n",
       "dtype: int8"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "008d3bea-16ae-4164-afbc-2975fd78c932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['a', 'b', 'c', 'd'], dtype='object')"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "c73c0fc1-5db8-4506-86da-0470b5fc0a0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_categories =['a','b','c','d','e']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "c26378e0-bb4a-4b87-9907-36703a3d0a2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s2 = cat_s.cat.set_categories(actual_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "id": "ee3ec1d8-f9a8-4a36-bdb6-1b063677209a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "2    c\n",
       "3    d\n",
       "4    a\n",
       "5    b\n",
       "6    c\n",
       "7    d\n",
       "dtype: category\n",
       "Categories (5, object): ['a', 'b', 'c', 'd', 'e']"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "8cf14617-9731-4232-96e2-d8867d409ba4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    2\n",
       "b    2\n",
       "c    2\n",
       "d    2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "aee805ab-ecc8-4a85-8541-72d6627483a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    2\n",
       "b    2\n",
       "c    2\n",
       "d    2\n",
       "e    0\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s2.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "38495066-d575-46f4-9e8e-14c0945539bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s3 = cat_s[cat_s.isin(['a','b'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "id": "b0e68a1b-c7bf-498a-8a1a-d2d3d80a65c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "4    a\n",
       "5    b\n",
       "dtype: category\n",
       "Categories (4, object): ['a', 'b', 'c', 'd']"
      ]
     },
     "execution_count": 267,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "id": "86736718-34ec-40d1-bf80-e204b3125521",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "4    a\n",
       "5    b\n",
       "dtype: category\n",
       "Categories (2, object): ['a', 'b']"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3.cat.remove_unused_categories()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "e13130f4-8ba9-4106-bf53-7a2d3e963a4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cats=pd.Series(['a','b','c','d']*2,dtype='category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "a346411d-0ed3-49e1-b11c-23a18d2420f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a    b    c    d\n",
       "0  1.0  0.0  0.0  0.0\n",
       "1  0.0  1.0  0.0  0.0\n",
       "2  0.0  0.0  1.0  0.0\n",
       "3  0.0  0.0  0.0  1.0\n",
       "4  1.0  0.0  0.0  0.0\n",
       "5  0.0  1.0  0.0  0.0\n",
       "6  0.0  0.0  1.0  0.0\n",
       "7  0.0  0.0  0.0  1.0"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(cat_s,dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d30967e0-ff34-41e5-9eb1-2c93e8708ee2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
